# -*- coding: utf-8 -*-
'''
Created on 24-09-2012

@author: LONG HOANG GIANG
'''

from urlparse import urljoin
import datetime
import os
import re
import sqlite3 as db
import sys
sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
from CrawlerLib import commonlib, html2text, AESCipher

def getListLink(url):
    tree = commonlib.loadweb(url).build_tree()
    if tree == None:
        print '::Build tree error'
        return
    data = []
    for item in tree.xpath("//div[contains(@id, 'book-navigation')]/ul/li/a"):
        href = item.get('href', '')
        if href == '': continue
        href = urljoin(url, href)
        title = commonlib.stringify(item)
        if '-' in title:
            title = commonlib.extractText(r"-([^-]+)", title, 1).strip()
        data.append({'url': href, 'title': title})
    return data

def getContent(url):
    tree = commonlib.loadweb(url).build_tree()
    contentNode = tree.xpath('''//div[contains(@class, 'content node-')]/div[1]''')
    if len(contentNode) == 0: raise Exception, 'sai xpath'
    html = html2text.html2text(commonlib.Etree.tostring(contentNode[0]))
    html = re.sub(r"\n", " <br />", html)
    return html

def createTable(c):
    c.execute("DROP TABLE IF EXISTS tblChapters")
    c.execute("DROP TABLE IF EXISTS tblDetail")
    c.execute("CREATE TABLE tblChapters(`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `title` TEXT)")
    c.execute("CREATE INDEX chapter_idx ON tblChapters(id)")
    c.execute("CREATE TABLE tblDetail(`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `chapterId` INTEGER, `detail` TEXT)")
    c.execute("CREATE INDEX detail_idx ON tblDetail(chapterId)")

def save2Db(c, title, content):
    if title.strip() == '' or (content.strip() == ''): return
    c.execute("INSERT INTO tblChapters(title) VALUES(?)", [cipher.encrypt(commonlib.toUpper(title))])
    c.execute("INSERT INTO tblDetail(chapterId, detail) VALUES(?, ?)", (c.lastrowid, content))

def process(url):
    
    conn = db.connect(DBNAME)
    c = conn.cursor()
    createTable(c)
    
    save2Db(c, 'Giới Thiệu', getContent(url))
    links = getListLink(url)
    for item in links:
        print '> ' + item['title']
        save2Db(c, item['title'], getContent(item['url']))
    c.close()
    conn.commit()
    conn.close()
    
if __name__ == '__main__':
    
    DBNAME = 'alobook'
    cipher = AESCipher.AESCipher('longhoanggiang')
    url = 'http://alobooks.vn/forum/14421/vap-nga-tuoi-hai-muoi-full-lam-phuong-lam.html'
    process(url)
    
    print '> Finished at {0}'.format(datetime.datetime.now())
    os._exit(1)
        
    